if [ -e /dev/tcaicard0 ];then
        AICARD_NUM=`ls /dev/tcaicard* | wc -l`
else
        echo "No card !"
        exit 1
fi



PSY_AICARD_NUM_1=`expr $AICARD_NUM / 3`
let PSY_AICARD_NUM=${PSY_AICARD_NUM_1}-1

CURTIME=`date +%m%d%H%M`
echo $CURTIME

SAVEDLOG=3CORElog/$CURTIME

CUR_PATH=`pwd`
log_file=$CUR_PATH/$SAVEDLOG/3COREFUN.log_all

mkdir -p $CUR_PATH/$SAVEDLOG
if [ -e $log_file ];then
    rm -rf $log_file
fi

echo $log_file

touch $log_file

if [[ $PSY_AICARD_NUM_1 < 4 ]];
then
	echo "unknown card!"
	echo "card num is $PSY_AICARD_NUM_1"
	echo "test FAILED"
	exit 1
fi

sh test_pci.sh
res=$(sh test_pci.sh | grep -c 'Speed 16GT/s, Width x16')


length=$(expr length "$res")
echo $res
if [[ $res -ne 8 ]];
then
        echo -e "ALL CARD CHECK PCI RESULT: \E[31m [FAILED] \E[0m" | tee -a $log_file
        exit

else
        echo -e "ALL CARD CHECK PCI RESULT: \E[32m [PASS] \E[0m" | tee -a $log_file
fi

sh check_all_3core.sh | tee -a $log_file


result=$(sh check_all_3core.sh | grep -c 'N/A')


length=$(expr length "$result")
echo "result is $result"

if [[ $result -ne 0 ]];
then
        echo -e "ALL CARD CHECK SN RESULT: \E[31m [FAILED] \E[0m" | tee -a $log_file
	exit

	echo "111"
else
        echo -e "ALL CARD CHECK SN RESULT: \E[32m [PASS] \E[0m" | tee -a $log_file
fi

echo "run tccl"
cd torchTest
SDAA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11 python3 ddp_autotest.py | tee -a $log_file

res=$(cat test_log/result_torch.txt | grep 'failed')
length=$(expr length "$res")
echo $length
if [[ length -gt 0 ]];
then
        echo -e "TCCL_FUNC TEST RESULT: \E[31m [FAILED] \E[0m" | tee -a $log_file
else
        echo -e "TCCL TEST RESULT: \E[32m [PASS] \E[0m" | tee -a $log_file
fi
cd -

#cd torch

for card_id in `seq 0 ${PSY_AICARD_NUM}`
do
    start_device=`expr $card_id \* 3`
    ./mem ${start_device}  | tee -a $log_file
done

for card_id in `seq 0 ${PSY_AICARD_NUM}`
do
    sh hbm_3core.sh $card_id $CURTIME --card_id_$card_id | tee -a $log_file &
done


wait

result=$(cat $log_file | grep  'FAILED')
length=$(expr length "$result")
if [[ length -gt 0 ]];
then
        echo -e "CARD_FUNC TEST RESULT: \E[31m [FAILED] \E[0m" | tee -a $log_file
else
        echo -e "CARD_FUNC TEST RESULT: \E[32m [PASS] \E[0m" | tee -a $log_file
fi


echo "function finished"
